{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Use OffPAC to Play Acrobot-v1\n",
    "\n",
    "PyTorch version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import sys\n",
    "import logging\n",
    "import itertools\n",
    "\n",
    "import numpy as np\n",
    "np.random.seed(0)\n",
    "import pandas as pd\n",
    "import gym\n",
    "import matplotlib.pyplot as plt\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.distributions as distributions\n",
    "torch.manual_seed(0)\n",
    "\n",
    "logging.basicConfig(level=logging.DEBUG,\n",
    "        format='%(asctime)s [%(levelname)s] %(message)s',\n",
    "        stream=sys.stdout, datefmt='%H:%M:%S')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22:37:18 [INFO] env: <AcrobotEnv<Acrobot-v1>>\n",
      "22:37:18 [INFO] action_space: Discrete(3)\n",
      "22:37:18 [INFO] observation_space: Box(-28.274333953857422, 28.274333953857422, (6,), float32)\n",
      "22:37:18 [INFO] reward_range: (-inf, inf)\n",
      "22:37:18 [INFO] metadata: {'render.modes': ['human', 'rgb_array'], 'video.frames_per_second': 15}\n",
      "22:37:18 [INFO] _max_episode_steps: 500\n",
      "22:37:18 [INFO] _elapsed_steps: None\n",
      "22:37:18 [INFO] id: Acrobot-v1\n",
      "22:37:18 [INFO] entry_point: gym.envs.classic_control:AcrobotEnv\n",
      "22:37:18 [INFO] reward_threshold: -100.0\n",
      "22:37:18 [INFO] nondeterministic: False\n",
      "22:37:18 [INFO] max_episode_steps: 500\n",
      "22:37:18 [INFO] _kwargs: {}\n",
      "22:37:18 [INFO] _env_name: Acrobot\n"
     ]
    }
   ],
   "source": [
    "env = gym.make('Acrobot-v1')\n",
    "env.seed(0)\n",
    "for key in vars(env):\n",
    "    logging.info('%s: %s', key, vars(env)[key])\n",
    "for key in vars(env.spec):\n",
    "    logging.info('%s: %s', key, vars(env.spec)[key])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class OffPACAgent:\n",
    "    def __init__(self, env):\n",
    "        self.action_n = env.action_space.n\n",
    "        self.gamma = 0.99\n",
    "\n",
    "        self.actor_net = self.build_net(\n",
    "                input_size=env.observation_space.shape[0],\n",
    "                hidden_sizes=[100,],\n",
    "                output_size=env.action_space.n, output_activator=nn.Softmax(1))\n",
    "        self.actor_optimizer = optim.Adam(self.actor_net.parameters(), 0.0002)\n",
    "        self.critic_net = self.build_net(\n",
    "                input_size=env.observation_space.shape[0],\n",
    "                hidden_sizes=[100,], output_size=self.action_n)\n",
    "        self.critic_optimizer = optim.Adam(self.critic_net.parameters(), 0.0004)\n",
    "        self.critic_loss = nn.MSELoss()\n",
    "\n",
    "    def build_net(self, input_size, hidden_sizes, output_size,\n",
    "            output_activator=None):\n",
    "        layers = []\n",
    "        for input_size, output_size in zip(\n",
    "                [input_size,] + hidden_sizes, hidden_sizes + [output_size,]):\n",
    "            layers.append(nn.Linear(input_size, output_size))\n",
    "            layers.append(nn.ReLU())\n",
    "        layers = layers[:-1]\n",
    "        if output_activator:\n",
    "            layers.append(output_activator)\n",
    "        net = nn.Sequential(*layers)\n",
    "        return net\n",
    "\n",
    "    def reset(self, mode=None):\n",
    "        self.mode = mode\n",
    "        if self.mode == 'train':\n",
    "            self.trajectory = []\n",
    "            self.discount = 1.\n",
    "\n",
    "    def step(self, observation, reward, done):\n",
    "        if self.mode == 'train':\n",
    "            action = np.random.choice(self.action_n)\n",
    "            self.trajectory += [observation, reward, done, action]\n",
    "            if len(self.trajectory) >= 8:\n",
    "                self.learn()\n",
    "            self.discount *= self.gamma\n",
    "        else:\n",
    "            state_tensor = torch.as_tensor(observation, dtype=torch.float).unsqueeze(0)\n",
    "            prob_tensor = self.actor_net(state_tensor)\n",
    "            action_tensor = distributions.Categorical(prob_tensor).sample()\n",
    "            action = action_tensor.numpy()[0]\n",
    "        return action\n",
    "\n",
    "    def close(self):\n",
    "        pass\n",
    "\n",
    "    def learn(self):\n",
    "        state, _, _, action, next_state, reward, done, next_action = \\\n",
    "                self.trajectory[-8:]\n",
    "        state_tensor = torch.as_tensor(state, dtype=torch.float).unsqueeze(0)\n",
    "        next_state_tensor = torch.as_tensor(state, dtype=torch.float).unsqueeze(0)\n",
    "\n",
    "        # train actor\n",
    "        q_tensor = self.critic_net(state_tensor)[0, action]\n",
    "        pi_tensor = self.actor_net(state_tensor)[0, action]\n",
    "        behavior_prob = 1. / self.action_n\n",
    "        actor_loss_tensor = -self.discount * q_tensor / behavior_prob * pi_tensor\n",
    "        self.actor_optimizer.zero_grad()\n",
    "        actor_loss_tensor.backward()\n",
    "        self.actor_optimizer.step()\n",
    "\n",
    "        # train critic\n",
    "        next_q_tensor = self.critic_net(next_state_tensor)[:, next_action]\n",
    "        target_tensor = reward + (1. - done) * self.gamma * next_q_tensor\n",
    "        pred_tensor = self.critic_net(state_tensor)[:, action]\n",
    "        critic_loss_tensor = self.critic_loss(pred_tensor, target_tensor)\n",
    "        pi_tensor = self.actor_net(state_tensor)[0, action]\n",
    "        ratio_tensor = pi_tensor / behavior_prob # importance sampling ratio\n",
    "        critic_loss_tensor *= ratio_tensor\n",
    "        self.critic_optimizer.zero_grad()\n",
    "        critic_loss_tensor.backward()\n",
    "        self.critic_optimizer.step()\n",
    "\n",
    "\n",
    "agent = OffPACAgent(env)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22:37:18 [INFO] ==== train ====\n",
      "22:37:20 [DEBUG] train episode 0: reward = -500.00, steps = 500\n",
      "22:37:22 [DEBUG] train episode 1: reward = -431.00, steps = 432\n",
      "22:37:24 [DEBUG] train episode 2: reward = -500.00, steps = 500\n",
      "22:37:26 [DEBUG] train episode 3: reward = -500.00, steps = 500\n",
      "22:37:28 [DEBUG] train episode 4: reward = -466.00, steps = 467\n",
      "22:37:30 [DEBUG] train episode 5: reward = -500.00, steps = 500\n",
      "22:37:31 [DEBUG] train episode 6: reward = -348.00, steps = 349\n",
      "22:37:33 [DEBUG] train episode 7: reward = -325.00, steps = 326\n",
      "22:37:35 [DEBUG] train episode 8: reward = -500.00, steps = 500\n",
      "22:37:36 [DEBUG] train episode 9: reward = -500.00, steps = 500\n",
      "22:37:38 [DEBUG] train episode 10: reward = -457.00, steps = 458\n",
      "22:37:40 [DEBUG] train episode 11: reward = -356.00, steps = 357\n",
      "22:37:42 [DEBUG] train episode 12: reward = -500.00, steps = 500\n",
      "22:37:43 [DEBUG] train episode 13: reward = -500.00, steps = 500\n",
      "22:37:45 [DEBUG] train episode 14: reward = -480.00, steps = 481\n",
      "22:37:47 [DEBUG] train episode 15: reward = -500.00, steps = 500\n",
      "22:37:49 [DEBUG] train episode 16: reward = -500.00, steps = 500\n",
      "22:37:50 [DEBUG] train episode 17: reward = -500.00, steps = 500\n",
      "22:37:52 [DEBUG] train episode 18: reward = -500.00, steps = 500\n",
      "22:37:53 [DEBUG] train episode 19: reward = -500.00, steps = 500\n",
      "22:37:55 [DEBUG] train episode 20: reward = -500.00, steps = 500\n",
      "22:37:57 [DEBUG] train episode 21: reward = -500.00, steps = 500\n",
      "22:37:58 [DEBUG] train episode 22: reward = -500.00, steps = 500\n",
      "22:38:00 [DEBUG] train episode 23: reward = -500.00, steps = 500\n",
      "22:38:02 [DEBUG] train episode 24: reward = -500.00, steps = 500\n",
      "22:38:04 [DEBUG] train episode 25: reward = -500.00, steps = 500\n",
      "22:38:06 [DEBUG] train episode 26: reward = -500.00, steps = 500\n",
      "22:38:08 [DEBUG] train episode 27: reward = -490.00, steps = 491\n",
      "22:38:10 [DEBUG] train episode 28: reward = -500.00, steps = 500\n",
      "22:38:12 [DEBUG] train episode 29: reward = -500.00, steps = 500\n",
      "22:38:14 [DEBUG] train episode 30: reward = -452.00, steps = 453\n",
      "22:38:16 [DEBUG] train episode 31: reward = -500.00, steps = 500\n",
      "22:38:18 [DEBUG] train episode 32: reward = -355.00, steps = 356\n",
      "22:38:20 [DEBUG] train episode 33: reward = -500.00, steps = 500\n",
      "22:38:22 [DEBUG] train episode 34: reward = -500.00, steps = 500\n",
      "22:38:24 [DEBUG] train episode 35: reward = -331.00, steps = 332\n",
      "22:38:26 [DEBUG] train episode 36: reward = -500.00, steps = 500\n",
      "22:38:28 [DEBUG] train episode 37: reward = -473.00, steps = 474\n",
      "22:38:29 [DEBUG] train episode 38: reward = -253.00, steps = 254\n",
      "22:38:31 [DEBUG] train episode 39: reward = -353.00, steps = 354\n",
      "22:38:33 [DEBUG] train episode 40: reward = -500.00, steps = 500\n",
      "22:38:35 [DEBUG] train episode 41: reward = -394.00, steps = 395\n",
      "22:38:37 [DEBUG] train episode 42: reward = -196.00, steps = 197\n",
      "22:38:39 [DEBUG] train episode 43: reward = -374.00, steps = 375\n",
      "22:38:40 [DEBUG] train episode 44: reward = -357.00, steps = 358\n",
      "22:38:42 [DEBUG] train episode 45: reward = -231.00, steps = 232\n",
      "22:38:44 [DEBUG] train episode 46: reward = -451.00, steps = 452\n",
      "22:38:46 [DEBUG] train episode 47: reward = -500.00, steps = 500\n",
      "22:38:48 [DEBUG] train episode 48: reward = -291.00, steps = 292\n",
      "22:38:50 [DEBUG] train episode 49: reward = -500.00, steps = 500\n",
      "22:38:52 [DEBUG] train episode 50: reward = -500.00, steps = 500\n",
      "22:38:54 [DEBUG] train episode 51: reward = -500.00, steps = 500\n",
      "22:38:56 [DEBUG] train episode 52: reward = -500.00, steps = 500\n",
      "22:38:58 [DEBUG] train episode 53: reward = -500.00, steps = 500\n",
      "22:39:00 [DEBUG] train episode 54: reward = -500.00, steps = 500\n",
      "22:39:03 [DEBUG] train episode 55: reward = -500.00, steps = 500\n",
      "22:39:05 [DEBUG] train episode 56: reward = -500.00, steps = 500\n",
      "22:39:07 [DEBUG] train episode 57: reward = -433.00, steps = 434\n",
      "22:39:09 [DEBUG] train episode 58: reward = -500.00, steps = 500\n",
      "22:39:11 [DEBUG] train episode 59: reward = -364.00, steps = 365\n",
      "22:39:13 [DEBUG] train episode 60: reward = -482.00, steps = 483\n",
      "22:39:14 [DEBUG] train episode 61: reward = -500.00, steps = 500\n",
      "22:39:16 [DEBUG] train episode 62: reward = -500.00, steps = 500\n",
      "22:39:18 [DEBUG] train episode 63: reward = -500.00, steps = 500\n",
      "22:39:20 [DEBUG] train episode 64: reward = -489.00, steps = 490\n",
      "22:39:22 [DEBUG] train episode 65: reward = -500.00, steps = 500\n",
      "22:39:24 [DEBUG] train episode 66: reward = -417.00, steps = 418\n",
      "22:39:26 [DEBUG] train episode 67: reward = -500.00, steps = 500\n",
      "22:39:28 [DEBUG] train episode 68: reward = -308.00, steps = 309\n",
      "22:39:30 [DEBUG] train episode 69: reward = -492.00, steps = 493\n",
      "22:39:32 [DEBUG] train episode 70: reward = -234.00, steps = 235\n",
      "22:39:34 [DEBUG] train episode 71: reward = -249.00, steps = 250\n",
      "22:39:36 [DEBUG] train episode 72: reward = -500.00, steps = 500\n",
      "22:39:38 [DEBUG] train episode 73: reward = -264.00, steps = 265\n",
      "22:39:40 [DEBUG] train episode 74: reward = -273.00, steps = 274\n",
      "22:39:42 [DEBUG] train episode 75: reward = -154.00, steps = 155\n",
      "22:39:44 [DEBUG] train episode 76: reward = -136.00, steps = 137\n",
      "22:39:46 [DEBUG] train episode 77: reward = -246.00, steps = 247\n",
      "22:39:48 [DEBUG] train episode 78: reward = -201.00, steps = 202\n",
      "22:39:50 [DEBUG] train episode 79: reward = -348.00, steps = 349\n",
      "22:39:52 [DEBUG] train episode 80: reward = -159.00, steps = 160\n",
      "22:39:54 [DEBUG] train episode 81: reward = -361.00, steps = 362\n",
      "22:39:55 [DEBUG] train episode 82: reward = -131.00, steps = 132\n",
      "22:39:58 [DEBUG] train episode 83: reward = -177.00, steps = 178\n",
      "22:39:59 [DEBUG] train episode 84: reward = -203.00, steps = 204\n",
      "22:40:01 [DEBUG] train episode 85: reward = -146.00, steps = 147\n",
      "22:40:03 [DEBUG] train episode 86: reward = -176.00, steps = 177\n",
      "22:40:04 [DEBUG] train episode 87: reward = -232.00, steps = 233\n",
      "22:40:06 [DEBUG] train episode 88: reward = -237.00, steps = 238\n",
      "22:40:08 [DEBUG] train episode 89: reward = -297.00, steps = 298\n",
      "22:40:10 [DEBUG] train episode 90: reward = -269.00, steps = 270\n",
      "22:40:12 [DEBUG] train episode 91: reward = -198.00, steps = 199\n",
      "22:40:13 [DEBUG] train episode 92: reward = -192.00, steps = 193\n",
      "22:40:15 [DEBUG] train episode 93: reward = -384.00, steps = 385\n",
      "22:40:17 [DEBUG] train episode 94: reward = -307.00, steps = 308\n",
      "22:40:19 [DEBUG] train episode 95: reward = -223.00, steps = 224\n",
      "22:40:20 [DEBUG] train episode 96: reward = -225.00, steps = 226\n",
      "22:40:22 [DEBUG] train episode 97: reward = -282.00, steps = 283\n",
      "22:40:24 [DEBUG] train episode 98: reward = -151.00, steps = 152\n",
      "22:40:26 [DEBUG] train episode 99: reward = -233.00, steps = 234\n",
      "22:40:28 [DEBUG] train episode 100: reward = -240.00, steps = 241\n",
      "22:40:29 [DEBUG] train episode 101: reward = -197.00, steps = 198\n",
      "22:40:31 [DEBUG] train episode 102: reward = -191.00, steps = 192\n",
      "22:40:33 [DEBUG] train episode 103: reward = -172.00, steps = 173\n",
      "22:40:34 [DEBUG] train episode 104: reward = -176.00, steps = 177\n",
      "22:40:36 [DEBUG] train episode 105: reward = -137.00, steps = 138\n",
      "22:40:38 [DEBUG] train episode 106: reward = -191.00, steps = 192\n",
      "22:40:39 [DEBUG] train episode 107: reward = -163.00, steps = 164\n",
      "22:40:41 [DEBUG] train episode 108: reward = -153.00, steps = 154\n",
      "22:40:43 [DEBUG] train episode 109: reward = -124.00, steps = 125\n",
      "22:40:44 [DEBUG] train episode 110: reward = -127.00, steps = 128\n",
      "22:40:46 [DEBUG] train episode 111: reward = -130.00, steps = 131\n",
      "22:40:48 [DEBUG] train episode 112: reward = -91.00, steps = 92\n",
      "22:40:49 [DEBUG] train episode 113: reward = -136.00, steps = 137\n",
      "22:40:51 [DEBUG] train episode 114: reward = -133.00, steps = 134\n",
      "22:40:53 [DEBUG] train episode 115: reward = -121.00, steps = 122\n",
      "22:40:54 [DEBUG] train episode 116: reward = -107.00, steps = 108\n",
      "22:40:56 [DEBUG] train episode 117: reward = -139.00, steps = 140\n",
      "22:40:57 [DEBUG] train episode 118: reward = -118.00, steps = 119\n",
      "22:40:59 [DEBUG] train episode 119: reward = -87.00, steps = 88\n",
      "22:40:59 [INFO] ==== test ====\n",
      "22:40:59 [DEBUG] test episode 0: reward = -131.00, steps = 132\n",
      "22:40:59 [DEBUG] test episode 1: reward = -118.00, steps = 119\n",
      "22:40:59 [DEBUG] test episode 2: reward = -118.00, steps = 119\n",
      "22:40:59 [DEBUG] test episode 3: reward = -150.00, steps = 151\n",
      "22:41:00 [DEBUG] test episode 4: reward = -148.00, steps = 149\n",
      "22:41:00 [DEBUG] test episode 5: reward = -128.00, steps = 129\n",
      "22:41:00 [DEBUG] test episode 6: reward = -119.00, steps = 120\n",
      "22:41:00 [DEBUG] test episode 7: reward = -135.00, steps = 136\n",
      "22:41:00 [DEBUG] test episode 8: reward = -103.00, steps = 104\n",
      "22:41:00 [DEBUG] test episode 9: reward = -103.00, steps = 104\n",
      "22:41:00 [DEBUG] test episode 10: reward = -114.00, steps = 115\n",
      "22:41:00 [DEBUG] test episode 11: reward = -122.00, steps = 123\n",
      "22:41:00 [DEBUG] test episode 12: reward = -105.00, steps = 106\n",
      "22:41:00 [DEBUG] test episode 13: reward = -145.00, steps = 146\n",
      "22:41:00 [DEBUG] test episode 14: reward = -135.00, steps = 136\n",
      "22:41:00 [DEBUG] test episode 15: reward = -112.00, steps = 113\n",
      "22:41:00 [DEBUG] test episode 16: reward = -99.00, steps = 100\n",
      "22:41:01 [DEBUG] test episode 17: reward = -144.00, steps = 145\n",
      "22:41:01 [DEBUG] test episode 18: reward = -111.00, steps = 112\n",
      "22:41:01 [DEBUG] test episode 19: reward = -136.00, steps = 137\n",
      "22:41:01 [DEBUG] test episode 20: reward = -121.00, steps = 122\n",
      "22:41:01 [DEBUG] test episode 21: reward = -130.00, steps = 131\n",
      "22:41:01 [DEBUG] test episode 22: reward = -109.00, steps = 110\n",
      "22:41:01 [DEBUG] test episode 23: reward = -154.00, steps = 155\n",
      "22:41:01 [DEBUG] test episode 24: reward = -90.00, steps = 91\n",
      "22:41:01 [DEBUG] test episode 25: reward = -102.00, steps = 103\n",
      "22:41:01 [DEBUG] test episode 26: reward = -123.00, steps = 124\n",
      "22:41:01 [DEBUG] test episode 27: reward = -93.00, steps = 94\n",
      "22:41:02 [DEBUG] test episode 28: reward = -151.00, steps = 152\n",
      "22:41:02 [DEBUG] test episode 29: reward = -106.00, steps = 107\n",
      "22:41:02 [DEBUG] test episode 30: reward = -126.00, steps = 127\n",
      "22:41:02 [DEBUG] test episode 31: reward = -113.00, steps = 114\n",
      "22:41:02 [DEBUG] test episode 32: reward = -128.00, steps = 129\n",
      "22:41:02 [DEBUG] test episode 33: reward = -104.00, steps = 105\n",
      "22:41:02 [DEBUG] test episode 34: reward = -111.00, steps = 112\n",
      "22:41:02 [DEBUG] test episode 35: reward = -92.00, steps = 93\n",
      "22:41:02 [DEBUG] test episode 36: reward = -93.00, steps = 94\n",
      "22:41:02 [DEBUG] test episode 37: reward = -112.00, steps = 113\n",
      "22:41:02 [DEBUG] test episode 38: reward = -135.00, steps = 136\n",
      "22:41:02 [DEBUG] test episode 39: reward = -118.00, steps = 119\n",
      "22:41:02 [DEBUG] test episode 40: reward = -108.00, steps = 109\n",
      "22:41:03 [DEBUG] test episode 41: reward = -147.00, steps = 148\n",
      "22:41:03 [DEBUG] test episode 42: reward = -117.00, steps = 118\n",
      "22:41:03 [DEBUG] test episode 43: reward = -184.00, steps = 185\n",
      "22:41:03 [DEBUG] test episode 44: reward = -113.00, steps = 114\n",
      "22:41:03 [DEBUG] test episode 45: reward = -98.00, steps = 99\n",
      "22:41:03 [DEBUG] test episode 46: reward = -132.00, steps = 133\n",
      "22:41:03 [DEBUG] test episode 47: reward = -94.00, steps = 95\n",
      "22:41:03 [DEBUG] test episode 48: reward = -100.00, steps = 101\n",
      "22:41:03 [DEBUG] test episode 49: reward = -133.00, steps = 134\n",
      "22:41:03 [DEBUG] test episode 50: reward = -125.00, steps = 126\n",
      "22:41:03 [DEBUG] test episode 51: reward = -163.00, steps = 164\n",
      "22:41:03 [DEBUG] test episode 52: reward = -179.00, steps = 180\n",
      "22:41:04 [DEBUG] test episode 53: reward = -152.00, steps = 153\n",
      "22:41:04 [DEBUG] test episode 54: reward = -263.00, steps = 264\n",
      "22:41:04 [DEBUG] test episode 55: reward = -103.00, steps = 104\n",
      "22:41:04 [DEBUG] test episode 56: reward = -110.00, steps = 111\n",
      "22:41:04 [DEBUG] test episode 57: reward = -159.00, steps = 160\n",
      "22:41:04 [DEBUG] test episode 58: reward = -157.00, steps = 158\n",
      "22:41:04 [DEBUG] test episode 59: reward = -138.00, steps = 139\n",
      "22:41:04 [DEBUG] test episode 60: reward = -124.00, steps = 125\n",
      "22:41:04 [DEBUG] test episode 61: reward = -117.00, steps = 118\n",
      "22:41:04 [DEBUG] test episode 62: reward = -184.00, steps = 185\n",
      "22:41:05 [DEBUG] test episode 63: reward = -120.00, steps = 121\n",
      "22:41:05 [DEBUG] test episode 64: reward = -111.00, steps = 112\n",
      "22:41:05 [DEBUG] test episode 65: reward = -116.00, steps = 117\n",
      "22:41:05 [DEBUG] test episode 66: reward = -122.00, steps = 123\n",
      "22:41:05 [DEBUG] test episode 67: reward = -135.00, steps = 136\n",
      "22:41:05 [DEBUG] test episode 68: reward = -138.00, steps = 139\n",
      "22:41:05 [DEBUG] test episode 69: reward = -117.00, steps = 118\n",
      "22:41:05 [DEBUG] test episode 70: reward = -123.00, steps = 124\n",
      "22:41:05 [DEBUG] test episode 71: reward = -96.00, steps = 97\n",
      "22:41:05 [DEBUG] test episode 72: reward = -113.00, steps = 114\n",
      "22:41:05 [DEBUG] test episode 73: reward = -90.00, steps = 91\n",
      "22:41:05 [DEBUG] test episode 74: reward = -116.00, steps = 117\n",
      "22:41:05 [DEBUG] test episode 75: reward = -132.00, steps = 133\n",
      "22:41:06 [DEBUG] test episode 76: reward = -117.00, steps = 118\n",
      "22:41:06 [DEBUG] test episode 77: reward = -104.00, steps = 105\n",
      "22:41:06 [DEBUG] test episode 78: reward = -142.00, steps = 143\n",
      "22:41:06 [DEBUG] test episode 79: reward = -124.00, steps = 125\n",
      "22:41:06 [DEBUG] test episode 80: reward = -92.00, steps = 93\n",
      "22:41:06 [DEBUG] test episode 81: reward = -132.00, steps = 133\n",
      "22:41:06 [DEBUG] test episode 82: reward = -121.00, steps = 122\n",
      "22:41:06 [DEBUG] test episode 83: reward = -93.00, steps = 94\n",
      "22:41:06 [DEBUG] test episode 84: reward = -103.00, steps = 104\n",
      "22:41:06 [DEBUG] test episode 85: reward = -198.00, steps = 199\n",
      "22:41:06 [DEBUG] test episode 86: reward = -190.00, steps = 191\n",
      "22:41:07 [DEBUG] test episode 87: reward = -142.00, steps = 143\n",
      "22:41:07 [DEBUG] test episode 88: reward = -134.00, steps = 135\n",
      "22:41:07 [DEBUG] test episode 89: reward = -126.00, steps = 127\n",
      "22:41:07 [DEBUG] test episode 90: reward = -144.00, steps = 145\n",
      "22:41:07 [DEBUG] test episode 91: reward = -113.00, steps = 114\n",
      "22:41:07 [DEBUG] test episode 92: reward = -103.00, steps = 104\n",
      "22:41:07 [DEBUG] test episode 93: reward = -117.00, steps = 118\n",
      "22:41:07 [DEBUG] test episode 94: reward = -142.00, steps = 143\n",
      "22:41:07 [DEBUG] test episode 95: reward = -132.00, steps = 133\n",
      "22:41:07 [DEBUG] test episode 96: reward = -115.00, steps = 116\n",
      "22:41:07 [DEBUG] test episode 97: reward = -102.00, steps = 103\n",
      "22:41:07 [DEBUG] test episode 98: reward = -86.00, steps = 87\n",
      "22:41:07 [DEBUG] test episode 99: reward = -137.00, steps = 138\n",
      "22:41:07 [INFO] average episode reward = -125.25 ± 26.61\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAD4CAYAAAAEhuazAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAABTmklEQVR4nO29eZQc13Xm+b2I3GvHvhMgCO6kRAmmSGqxZFEWrY1S21ZTXiS3281uj+T2emyrdaY93W6N51h9tI0t2bLclmVrGa0WLUu2RG3WRrFBijsBEAAJoIACUIXac4vtzR8vXsSLyBe5Z1Vl5v2dg4OqyKysiKjIL25+9757GeccBEEQxHBhrPcOEARBEGsPiT9BEMQQQuJPEAQxhJD4EwRBDCEk/gRBEENIar13oFm2bNnC9+/fv967QRAE0Vc89NBDc5zzrfHtfSP++/fvx5EjR9Z7NwiCIPoKxthp3XayfQiCIIYQEn+CIIghhMSfIAhiCCHxJwiCGEJI/AmCIIYQEn+CIIghhMSfIAhiCCHxJwiC2KAceW4ef/bNZ1CsOl1/bRJ/giCIDco3j17CB77xDNJm96WaxJ8gCKLLzBctfORfT6LTYVnHL67gyi2jyKRI/AmCIDY89z1yDv/3V47i+MXVjl7n6IUVXL1jrEt7FaUj8WeM/Txj7EnGmMcYOxx77J2MsROMsWOMsVcr21/IGHvcf+yDjDHWyT4QBEFsNJ67XAIAnF8st/0aq1UH0wtlXLN9tFu7FaHTyP8JAP8GwL+qGxlj1wO4B8ANAO4C8CHGmOk//GEA9wI45P+7q8N9IAiC2FCcmRfiP92B+D9zcQUAcPX2DRj5c86f5pwf0zx0N4BPc86rnPNnAZwAcCtjbCeAcc75D7kwwz4O4I2d7ANBEMRG4/TlIoDOIv/jvvhfsxFtnzrsBnBW+X7a37bb/zq+XQtj7F7G2BHG2JHZ2dme7ChBEEQ38TyOswtC9DsR/2MXVpFPm9g7VejWrkVo2M+fMXY/gB2ah97FOf9S0o9ptvE627Vwzj8C4CMAcPjw4c7S5gRBEGvAheUKLMcDAJxb6Czyv3r7KAyjN2nRhuLPOb+zjdedBrBX+X4PgPP+9j2a7QRBEAPBaT/Zu2M811Hkf/TCCl5xTc0Arq7RK9vnPgD3MMayjLEDEIndBznnMwBWGGO3+VU+bwWQ9OmBIAii7zgzL/z+2w9uxoXlCmzXS3zu7EoVr/nAd/Ho2cXI9surVcytVnvm9wOdl3q+iTE2DeB2AP/EGPsXAOCcPwngMwCeAvDPAN7OOXf9H/t1AB+FSAKfBPDVTvaBIAhiI3H6cgkpg+Hw/il4HLi4XEl87ucfnsZTM8v47jPRnKZcH9CrSh+gwxm+nPMvAvhiwmPvBvBuzfYjAG7s5PcSBEFsVE7Pl7BnKo99m0Si9vxiBXs0SVvOOT57RNTFxBeD9brSB6AVvgRBEF3l9OUi9m0ewa7JPADg3GJJ+7yHzyzi5GwRGdMIxF5y7OIKJgtpbBvL9mw/SfwJgiC6BOccpy+XsH9zAbsmhPifX9TbPp976CzyaRM/f3gPTs0W4Si5gWMXVnD19jH0sgECiT9BEESXWCzZWKk42LepgHzGxOaRDKY15Z4ly8E/PjqD19y0E7fsm4LlejjtrwrmnOP4hRVc00O/HyDxJwiC6BpSwK/YPAIA2DWZ15Z7/vMTF7BadfDmw3twaJvo3fOM7/ufnF3FStXBDbvGe7qvJP4EQRBdQrZ1uGKzSPDuThD/T/7oDK7YXMCtBzbhqkD8he//w5OXAYhS0V5C4k8QBNEirsfxu595FB/+9klUbDfYfsZf4CUrfXZN5nFusRzp6//I2UUcOb2At92+H4wxjGRT2DOVx/FLIvL/4anL2DWRC16jV3RU6kkQBDGMPDu3is8/LNqU/f0Dp/GHP3MtXnfzTpyeL2H7eBa5tGhivHsqj5LlYqlsY7KQAQD89feexVg2hTf/RNgE4ertY3jm4go8j+OBU/N4+TVbe5rsBSjyJwiCaJmnZoRF80evvx7j+TR+41M/xs9++Ad4+PQCrtg0Ejxv92QOAIKk77nFMr7y+Aze8qJ9GM2GsfehbaM4NVvEUzPLmC9auP3K3lo+AIk/QRBN8I2nL6JkdX+IeL/y1PllpE2GX3zRFfjyb7wEf/qzN+PsQhmn5orYtzm0a2Stv/T9P/b9ZwEAb7tjf+T1Dm0fg+V6+P/+t1j01Wu/HyDbhyCIBsyuVPHv//YI/vjuG/DLt+9f793ZEDw9s4yDW8PZum/+ib147c078dkjZ3HHVVuC5+0OFnqVcXG5gk8/eBavuWlnsF1ytT+t6wsPT2PfpoJ2RXC3IfEniCGBc96Wj1y2RELzbAftiQeNp2eW8RJF5AFgJJvCr7z4QGTbppEMcmkDn37wLN7zL8fguBz/8WVX1rzewa1C/IuWi9etgeUDkO1DEEPBlx45h9v/5JtBn/lWsPyVp+c6aE/cL1RsF3/5nZN1O3FeXq3i0koV1zdRh88Yw96pAo5dXMHLDm3F1377Zbhx90TN82TFD7A2lg9AkT9BDAWfe2gaF5YrWCrb2NpivxgphDNDIP7fPHoJf/LVo7hpzwTuOLhF+5yn/WTvdTubW4T1vn/7fFiuhxfsm6r7vKu3j2F6oUziTxBEd1itOvjRqXkAQLHqtC3+ST1qBonn/EVal5aric95amYJQPPir4v0dbz+eTsxkU9j+3iuqed3Cok/QQw433tmLrBuim1U7Ejxv7giBpOkzcF1i+UirXo9+J+eWcH28Sw2jWS6+rvfdMsevOmWPY2f2CUG969IEAQA4JtHLwZflyy3zjP12K5YncoTBpPYrheIZr8jRzBeWkmO/J+eWW466t/IkPgTxADjeRzfPDoblBauVtuP/AG99fP5h6Zx53u/g/mi1f6ObhDOzNcX/6rj4sSlVRJ/giA2No+fW8LcahWvf94uAECp2k7kr4p/bdL3ucslWK6Hp2eW29/RDUDVcXF+SRxfku3zzMVVOB7H9ST+BEFsZL5x9BIMBrz2pp0ARMK3VSwnbEomxVFl1o+S11P8lys27vnID4Oumu0wvVAG50DKYMExxZHHSJE/QRAbmm8evYgX7JsKasg7SfgC+sh/dlUI5bELKzWPrRWnZot44NQ8Hjq90PZryLzFDbsnEiP/R84uIpc2cGDLiPbxfoLEnyAGFMf18MS5Zdx+cDNG/CZi7UT+UvxHsymt5y+j5KPrKP5yFfJS2W77NeSnhlv3T6FkuTX5kQtLFXzuoWncdcMOmEZvO26uBST+BDGgFH1BnMinkUkZSJss2NYKUvyv2FzQR/6++B+/uALX4zWPrwWyp/5yuf3mc6fnSyhkzGDl7qVY9P+BbzwDj3P87k9f0/6ObiBI/AliQJHRsIz6C5kUSg0i//OLZdzy37+GE5fCKN7ySz33bx6pEX/X45gvVrF7Mo+q4wWLpNaaUhci/zOXS9i3qYBtY2KR1UVlodfJ2VV85shZ/OKLrsDeHg9ZWStI/AliQJH+fiEjBouMZlNYbVDtc2a+hIWSjefmwrp92+8HtG9zAcsVByuVUGAvF6vwOIImZ0dn1sf6KdtdsH3mS7hicwHbx8UK6EsrYeT/3q8dRzZl4O2vuKqzHd1AkPgTxIAiI/9CRkb+ZsOe/LLxm6UkeaXts9/vUz+zFIqitHzuuGozDAYcu7A+FT9l/7iWK+2Jv+dxnJkv4YrNI9jqR/6yxcOZyyX80+Mz+LWXHGi5NcZGhsSfIAYUmdyVkf9INtVwkVcg/k6t+F+xWVS4qNaPFP89U3kc2DKCp9cp6dtp5H9huQLL8bBvUwHjuRSyKSOI/H98VlQQ3XXjzu7s7AaBxJ8gBpSSLSN/Kf5mw/YOMuJXxV96/nKguFrxI8V/62gO1+4cX7dyT3lcy22Kv2zrcMXmAhhj2D6eC1b5PjWzjIxp4Kpto93Z2Q0CiT9BDChyNa+0fUYyqYalnlVH/Ew1ZvtkTAPbx3MwDRaN/P0a/y1jGVy7fQxn5ktttZDolE4j/zPzIlEt5+9uG8sGtf5PnV/Goe3h1K5BYbCOhiCIgJJVa/s0WuQlI35btX0cDymTwTQYdoznIqt8Z1eqGM2mUMikcK2/6nU9ov9KG5H/U+eX8e/+5kE8cW4Jpy+XkDIYdvkD17eNZ3FppQrOOZ48v4wbmhjc0m9QS2eCGFBKlsb2aVDtk5TwlW2cd03majx/mQS9dscYACH+L7yi/uCSbiOPtWi5Tbed/tSDZ/CtY7P4/snL2D6exe6pPFL+z20by+G7x+dwYbmC+aKFG3Y115O/n6DInyAGlFKszn8k0zjhW9UkfC2XB2K6cyJf4/lvHRXiv2cqj9FsCkfXoeJH2j5Ac9E/5xzfPn4JLzqwCbdfuRln58tBTgMQkf9K1cGR50SylyJ/giD6hpLlgDEg63vVhUwKVceD43pBhBtHJ/7C8xftDPZtKuArj8+gYrvIpU3MrlZx3Q4hjIwx7JnKr8vEr7KSyF6uONg8Wr8k87nLJZydL+Pel16JX3zRFfjcw9ORhK5c6PWtY5fA2GA0cotDkT9BDCgly8VIJgXGhHCPZIX9U6/FQ+D5x20f/wZy4+5xOB4P+viotg8AbB3LYnZlHcRfifybSfp+59glAMBPXr0NhsHw5sN7IzN25UKv7xybxYHNI8Gnp0GCxJ8gBpSS5SDv+/1AaP/UW+glvf6qEvk7iu1z855JAMBj04uo2C5WKtGZwNvGcontkHtJyXKRT4tjbUb8v318Fge2jGDfZn2rBhn5Xy5auG4ALR+AxJ8gBhYR+deKf71yT13C11ISqDsnctgymsFj00tKjX8s8l8VVTJrScV2sXNCCHYjz79iu3jg1GX85NVbE5+zTbmhDaLfD5D4E8TAUqy6yGdCu0LeCIp1Kn6SVvhKz58xhpv3TOKx6cWgxj8a+WdhuxyLpfZ77LRDyXKxfVyIf6PI/8Fn51GxvbriP1lII+Pf8Aax0gcg8SeIgaVsO/rIv47tIxd5xcVfLZ28afcETlxaDfrfxz1/oP4A9F5Qtl3smGhO/L9zfBaZlIEXXbkp8TmMseBYKPInCKKvEJG/Iv6ZVLA9CW3C1+ER8X/e3gl4HPj2sVkAtZE/gLZ8/4rt4ux8qfETdT9ruUG0Xq+5m+V4+OZRUeJZyNRP4m4bz2L7eBZbGlQO9SuDl8ImCAKAKH/c4VshAFDwq32aSfhG6/w9jGfSwfc37Z4EIMSfMWDTSCZ4TN4IZldbq/ip2C7e+r8exPGLK3jkv/50Sz/LOUfJdlHImBjPpxM9f845/vDzj+HZuSJ+r4mBLL9w6762Jp/1Cx1F/oyx9zDGjjLGHmOMfZExNqk89k7G2AnG2DHG2KuV7S9kjD3uP/ZBJuvQiKGn6rh48vzSeu/GwFC0nGB1LyD6+QOou9AraYWv9PwBIfC7JnJYKtvYVMhEPhVsG4+2Q24G1+P4nc88ggefncdiyW45WWy7HK7HUcikMJFPJdo+7/mXY/jCj8/hd151NV57c+MOnT9/eC9+5cUHWtqXfqJT2+frAG7knN8M4DiAdwIAY+x6APcAuAHAXQA+xBiTV+GHAdwL4JD/764O94EYEP7x0Rm84c++j6U1ThYOKmXLDaJ9IGzzUK/FgyzxrNbx/IGw5DPe334kYyKfNluyff7HPz2Frzx+AVduFU3V1BtPM8gFXrm0jPxrb25feXwGH/r2Sbzl1n34jZ8anIEsndCR+HPOv8Y5l2f6AQB7/K/vBvBpznmVc/4sgBMAbmWM7QQwzjn/IRe3948DeGMn+0AMDoslC67HsVIl8e8GIvIPnV35db3Iv6pd5MVrxP+mPaICJi7+jLGgKVozzK1W8Tfffw5vuXUffuHWfQCillMzyAVe+bSJiXxaG/l//8QcxnMp/PHdN4DMBkE3E76/CuCr/te7AZxVHpv2t+32v45v18IYu5cxdoQxdmR2draLu0psRHQLjIj2cD2Oiu1FbB/TYMinw2leRy8s42c+8N2IWOpKPS2nNvJ/nh/565KhW0ezTUf+c3656Euu2hK0TG717692L00S/+mFMvZtLiS2tRhGGp4Jxtj9jLEnNP/uVp7zLgAOgE/ITZqX4nW2a+Gcf4Rzfphzfnjr1uSaXGIwkIJTtUn8O6UcG+QiEW2dxWM/OjWPp2eWIxU2iXX+qehb96bdIvLfphlrKCL/5hK+C0Uh1FNKXX27kX8ubWI8l9ZW+5xbLGPP5GAMXu8WDat9OOd31nucMfY2AK8D8EoeZmqmAexVnrYHwHl/+x7NdoLQJhuJ9gij4ehbfCRrBhUs5/zWzBWlL05Q55/Q0lkyUUjjL37pBYH3r7J1NIvvrcw1tZ+LJQsAMFnI4II/PKVV8a8oN7oJv9rH8zgMQ9ywOOeYXijh5XUWdQ0jnVb73AXgDwC8gXOuFujeB+AexliWMXYAIrH7IOd8BsAKY+w2v8rnrQC+1Mk+EINDGPnX7zlPNCac4hWN/AuZVFDnH4p/tKwTiEf+tZ4/IGba7prM12zfOpbFcsWJ3FSSWPCT+5tGMoHt0+rNX7auzvvi73FgVSlnvVy0ULE97Jmq3ddhplMD7M8AjAH4OmPsEcbYXwAA5/xJAJ8B8BSAfwbwds65vBJ+HcBHIZLAJxHmCYghhzz/7hEOcolG/qNq5L9QG/nrFnlZTQ5HkcimaM34/gtB5N+B7WOFCd/xvDhetWJs2j/OPVNk+6h0tMiLc55YM8U5fzeAd2u2HwFwYye/lxhMdH4z0R7xEY6SQiYVWC1yIldZI/7yBsw5r6nzb0S40KuKvZvqC+5C0UI+bSKXNpH1u3K2evMPqn38yB9AxPefXhCmxG6K/CNQ6pvYMMSFh2ifcIpXVPxHs2KaV9Vxg3JMXeQv/3c9Ds7RUpVM0N+niYVeCyU7WCHcaeQvV/gC0f4+MvIn8Y9C4k9sGKqB7UOef6fIyD+fjn64L2RMlCwXF5bCapyKIrZVJekuon5Rw9Ga7RNG/o1YKFmYLAjB7tjz96t9gGhb53MLZUzk08FjhIDEn9gwkO3TPeLD2yUjfuR/ThnCXvGf63kcjseRMhg4BxyPB0KcbsH22TyahcGA2eXG5Z4LJQtTBRH5y3GTrSb81VLPwPZRVvlOL5Qo2auBxJ/YMJDt0z1kLX8hGxd/EfnLZC8Q2j5S6Edz4tOC7Yp5v0AYlTeDaTBsGsk2FfkvluyOI/+y5cLwZxVPFPS2D4l/LST+xIYhFH+yfTqlnFjnn4LrcTw7J3rxMwZU/PMtb7qyAZzleG3ZPoCwfprz/K2mPf8nzi3hwWfnaxq/lW0xwpExhtFMCgYLxV/U+Jep0kcDtXQmNgy2psacaA9Zyy/n2kpkT//jF1exdSyLUtVB2YredKPiL22f1sRfjnOsh+txLJVtTPq2TxD5J/z9//OnfoxTc0Xs21TAmw/vwX/6yYNImYYQf/+4DINhTFnlO1+0ULZdivw1UORPbBiozr97lG0XubQB04h69TIH8MylFeyezCOfMYPIX4quTIxWHa8tzx9oLvJfKtvgXLR2ABrbPpeLFl54xRS2jWXxP792HI9Oi/bfZctFPhNKmdrfJ6j00SxGG3ZI/IkNA3n+3aNYdYIoX0VG9WfmS9g9mUc2ZYaev3/eZXmo7YaRf6aNyH9utQrPS+7NLxd4xRO+usjf8ziWKzbuOLgZ//1usUxo1u8fVLZcFJSqJlX8ZWKbbJ9aSPyJDUOj9g4PnLqM759ormfMsCOiYbNme8EXf86BXZM55NKGJuGbDr63nfY9f8fjWKwzT1cuNpsaido+upv/quWAcyHsW8bE8+UK4pLtIqcc63g+FZR60gKvZEj8iQ2DWmOu4/33H8f7vn58LXdpQ2E5XtPJ8KKVFPmHIhnYPnY016J6/oHt00K1DwDsmBBi+4+PJvdtnFc6egL1E75SzMdzaWwe8UtJ5SI1y0VByW1M5NNYLIW2z3guFZSAEiEk/sSGIfD8E1o6F6vuUHf8vPfvjuA3P/VIU88tJUX+yg1h12QeOcX2qQaevy7h25rn/4prt+Ll12zFH933JP7yOye1z4nbPowxZExDG/lLG2c8n6opJS3ZTuRYb94ziVNzRXzxx9NU6VMHEn9iw9DI8y9ZzlBXAl1YquBrT11oqmFayXJrWjsAYVQP+OKf1nn+teLfquefTZn4yC8fxmtv3ok/+epRfPS7p2qeE7d9AGH96CN/Uboqk9Fbx8KBMWXLjVQ1/dpLDuDWA5vwX77wBB6bXqJKnwRI/IkNQyPxL1tupNvksOF4HB4HvvxY4xEYJcutae0ARFf87pkS4l9Osn3c9ks9ASHkH7znFjxv76TW/pkv2kibDCPKPmVSBiy31tqSpZuyd48q/hXbi0T+KdPAn73lFoxkU5hbrZLfnwCJP7FhsBr09inbw237yNW2//BIM+LvaCN/GdXLwSe5tBEk2INFXortY7WZ8JWYBsPBLSOYW7VqHlssWZgsZCIzdTNmUuQvxF969+qoyJLl1Kxn2Daewwff8nyYBsOhbWNt7fugQ4u8iA2B63G4fllgsu3jIpuqFbRhwXY5TIPh0bOLeHauiANbRhKfW7Lcmr4+gCinNA2G3ZN5MMYito+86Qaev+tBLqaNj3FshS1+2SfnPCL0oq9PNBGbZPssKQlfIFxExjlH2dYf6x0Ht+B7f/AKbNXMGSYo8ic2CJHhIZo3v+txVBUPehixXQ8vPbQFjAFfeuRc3eeWqk5NawdAJFULGTOYwJVLG0FjtHqef7uRPwBsGc2g6nhYrTqR7QslO0j2SoTto4n8Kw4YA8b8G9PWsSxsl2OhZKNie8il9UHBzok8DW1PgM4KsSFQo31d5F+O1aIPI47HsXeqgNsObMaXHjlf0+NGwjlHKSEaBoCDW0fxvL2TAET7h6DU0416/naHnr9kix95x62fhaJVI/7ZxISvjdFsKpjLK2cGyOHzScdKJEPiT2wIrIj413r+sj/9MFf72I6HlMnwxlt24dm5Ip6eWdE+r2ILu0YX+QPAF/+PO/Dbdx4CINogVxwXnPPg3I6pnn+bjd1UQvGPViktlGxMjdTaPrqb/3LZjvTjl1bOaV/8dWWtRH3I8yc2BFYD20dOaxpq28cTs3Sv2jYKIHlYStIIR4nqu+fSJjgXn7aqgfiHvX0M/7mtlnqqbB4V0f2cUqLKOQ8SvipJdf7LFTuo9AFqI/94wpdoDEX+RM95//3H8St/82Dd50jBz6dN7ZtfDifxOILE8LDhuBxpkwVRuJNwI0wa5KJDeuVV26tf6tlBwldG6XPF0PZZrTpwPI5NOs8/oc5/Ih/GqlL8z1ymyL9dSPyJnvPMxVU8eX657nMspcxQ19tHHTI+jNE/53LKloGUId62SechFP/GH+xzafFaFceF5XgwWHhD6FbCd9NIBoxFI/8Fv7XDZKzaJ9Hzr0Rtn/FcCpmUgdPzYi4BRf6tQ+JP9BzL9bBaceo/R/GbdUldafvI1xs2wqEqLCi7lNviFKXto6nzj5PzS2fLllhDkU2ZMA0G02B+wlf8jpTRfuSfMg1MFTIRzz/e2kGSVO2zVI7aPowxbB3N4uy86NpJkX/rkPgTPcd2PZRtN9GmABCs6hzLpVF1vJpKlpIq/kOY9HU8ccwps3HkL2+UhSaiYSmaMvKXnTXlYivb9ZA2WSRP0A5bRhPEf6TW80+q9ok3Z9s6lsX5JV/8KfJvGRJ/oudIkSpayR0pg2RjNgXOa6NamcRUX2+YUMcpyg6bTlLk79fTj2RbsH1s0TE0EP+UKv6dy8TmkSwuK6WeYeTfeJGX43ooWm7E9gGE+MsYoRmLi4hC4k/0HPlmji/y0T1HlhnGyz1V20f2mB8m1O6aad+Csb1ohdTbP/Ewnji3FORHmrFCpO1TsV1UHS+o6pH2i+3yroi/XOUrWQjaOcfr/M0a22fZtwzH81GBl0lfgCL/dqDbJdFzZK14Pd9fRrZqjblKacg9fyfw3o1AjG3lHM0slfFPj8/g1FwRv/CifQCg7ecfRw5BKdvC9smmVduHg7HuRP7C9gkj/8WSBcYQ8fEBv84/lvBfjrV2kKhtG8jzbx2K/ImeY7cQ+Y9mwxpzFbXaZxg9fzXyT/m99R2l5FU+/vTMMj7+g+cAtBb5V33xr4n8HQ+ZFnv569gymsVq1Qn6CC2UbEzm0zUzhnUJX9nRU+f5S0j8W4fEn+g5Upjqin+Q8JW2T0z8reEu9VRLLmUkroqk/OQ0kjHxzKVVAM3W+auevxfM0RWRv2ih3eoULx1b5EIv3/qZWapgs6bhWsY0YLs8Mvs3HORSR/zJ9mkZEn+i5wTiX8f20bUWUCkNufjLKD8VWeRVG/n/xisPIWWIiVjN2DWypl/aPrUJ3y55/rH+Pk/PLOPaHbWtluXvV29swSCXBM8/43cqJVqDxJ/oOVLIi03ZPgkJXzv82WH0/NXI3zQYGIveBOXX1+0cx9tfcRVu2D3e1OvKiLlih3X+4vcwWK6Y4dtV8V+pYqFo4dxiGTfsmqh5XlYn/pX6nj9F/e1BCV+i58iE70od8df1lVGJRv7DV+3jKIu8xP9G5DyEQ1cYfvtVV+O3/MZtjcgF4i/aO0hfPZMyYDscttklz38sbO721IxY7X3DrtobVBD5O2rkX9/zp46e7UHiT/Scpmwft3nbZ5gTvnKBV9pfgRt/XCZsm12UJSPtck3C18RS2UbaZV2q8w89/xX/OtCKv1kr/ktlG6bBakQ+lzYxlktR5N8mJP5EzwkXeTVh+9Sp8x/NprBadYbS87djrZXTKSOyYlquAG5VqA2DIeuXV0YWeckVvk3mDhqRS5sYy6Ywt2phsWRhx3hOm/CVpaaRyL9iYzyX0t7Qto5lSfzbhDx/oufIN/JKg4RvymDBG7lqxyN/J/jYP4ziH4q7EMCUYQR2GqDaPq2/peUoRzXhKxqsiTxAN6p9gHCh15Pnl7VRPwBkTP/vH7F9nBrLR7J3qoBNsRYRRHNQ5E/0FM/jQaVKozr/TMrQJvwAoGx7GM+ncW6xPNy2j7RlTBaJ/NV1AK0iRzmKhK8RvI5s6dwNzx8Q5Z7TC2WcnF3FXTfu0D5H5/nHm7qp/OnP3YyEgWZEAyjyJ3qK2oKgXrWP7Qrxl2/+eORftsJ+7sNZ7RPtrpkyDe0ir3YifznKsRor9bQd3rXePoCo+Hn83BI8rvf75e8FwnUfQG07Z5Xt4znsmMh1Zf+GDRJ/oqeoFSmNEr4Z0whKDeOef8lyQ9tnCCN/We0jxTHlR+aScOhK+7ZPXPy72dsHEOIvB/HoyjyBMOFbjVX7xGv8ic4h8Sd6iirUjUo9MykjSPjpVviGnn/t53zH9fAfPn4ED59Z6MZubzjCap9wrGLU9omWgrZCNm2GvX0CW8kUM3yd7kX+cpzjeC6FPVN57XO0pZ6VZM+faB8Sf6KnqNHpatVOfp4v/rrIj3OOkh2Kf9Kwj68/dRFHnpvv1q5vKOK2TspkkZtgvNSzFfJpI8jHZP2EezrFgpbOmQ5GOKrIhV7X7xpPLEXNJnn+CbYP0T4diT9j7I8ZY48xxh5hjH2NMbZLeeydjLETjLFjjLFXK9tfyBh73H/sg6zTKRHEhka+ibMpA8Vqcj9/WWMu3/yq+FuuB9fjdat9pBAO6gIw6e8HpZ6moa3zT7VZ7SMXUsmbR9Y0goRvN20fINnyAWrbO8gqpKSEL9E+nf5V38M5v5lz/nwAXwbwXwGAMXY9gHsA3ADgLgAfYozJYtwPA7gXwCH/310d7gOxgZGitHkk09jzTxlgjIm2vornX1Zm0qYMpq32kdt0w98HAScQd3+FrxEX//Ztn1zKxJLfP0f1/AGRa+mW+G8bl+Kf3HoiHvkHrR1I/LtOR39Vzrk6lXsEgAy77gbwac55lXP+LIATAG5ljO0EMM45/yEXc/o+DuCNnewDsbGRojRZyMByvZpErkRdXZqNjfKT7ZwLGVNUoWgif1kdMqhloLKmPy1X+KZYpLGbPG75eCvkM0rkHxP/quMFN5xOef6eSfzpz96M1968M/E5mdgnv7CXPyV8u03HZ5Qx9m4AbwWwBOAV/ubdAB5Qnjbtb7P9r+Pbk177XohPCdi3b1+nu0qsA1Ko5UKcYtUNKnpUIguM0kYkgpetHfIZs6anTfjz0vYZTPF3gmqecJHXqht+knI8sUjOaKO7ZS4d9tAP6/zDm0g7eQQdhsHw5p/YW/c58fYOS0FHT4r8u03Dvypj7H7G2BOaf3cDAOf8XZzzvQA+AeAd8sc0L8XrbNfCOf8I5/ww5/zw1q1bGx8NseGQIi4HdSdZP9L2AcQoP7XOX9o++bQQf13CV24b1Mg/aOlsqJ6/mvBtvyRTvRnHI3/5u9aKeLVPUkdPonMaRv6c8zubfK1PAvgnAH8EEdGrt/g9AM772/dothMDShD5+4O6VxIqfiK2T2yaU0nx/DOm3vOXv2dQI//A1gm6ekYbu1kd2DM5pTdO0NjNXGfxd6O2D5V6dp9Oq33UvrFvAHDU//o+APcwxrKMsQMQid0HOeczAFYYY7f5VT5vBfClTvaB2NhIgZpSbB8dauQfn+Na8hvC5et5/s5gRf5/+Z2T+MHJueB7x/P8Pv5hS+f4Ct927Rm1MZo+8l+7grx4qS95/r2j0zP6/zDGrgHgATgN4D8BAOf8ScbYZwA8BcAB8HbOuXw3/zqAjwHIA/iq/48YUOKef1Ktf7ypmOr5h9U+Zk2JY/Dz0vYZkMj/w985ibtu2IE7Dm4BIFb4qiKcMmtbOrcboctRjkBoAWUV8VdvBL2GMRZ0FAWAxZIf+Rco8u82HYk/5/xn6zz2bgDv1mw/AuDGTn4v0T/IN/FkQYh/UmdPS5kfm02ZkQi+FBN/mdzV/Z5Bifyr/nAVieV6kUqeeKmn4/IgGdwqOU3kn14n20fuQ5jwtZFPm9oiAaIzaIUv0VNkieKmQhO2j6nYPkpJaMlWEr4Jts+gef6W66EaE3fV068p9ewg8ldtn+w6J3zlPsjS3cWyjUmK+nsCiT/RU2Rvn05sn4pS6pmU8A0i/wEQf9fjcD0eqXhyvKi4i37+MdunjRp/IBygAiiev7k+nr/cB3nsiyWbkr09gsSf6CkyEpeebWKpp9JALJs2EmyfVMOEr62xhPoN3Y3McqKlnJmUEYn87S7ZPrrIv1t1/s2SUaq9lsoWRf49gsSf6ClSqLMpwx/DWGv7yIEvatQZWeRlO8ikDJgGS0z4ym3VAYj8g1YVSsWT40VLOVOaGb7dsH02hOdvRj1/ivx7A4k/0VOqQX26FP9a20dGeZFFXrHePnJ4t1jkVRvdVwco4VuVrSriCd2YIDseB+fhyub2q31q6/zVap9ujXFsFjXhu1iyMZmnMY29gMS/Sb7w8DSOXlhu/EQiglyFmk0ZGM2ltKMcpXBnFNsn3t6h4AuUiAprPz2EXT37X/zDyD8a2aeU1g3Sh1e7mbZrz+R0nv861fnL3225HjjnlPDtIST+TbBadfB7n30Un/zRmfXelb5D7UM/kmD7qG2fgejHfkBE/jk/8hee/2CXelY1nn88spdfy8Huttv+Ct9otU94niVr7vn7tl/FL3elGv/eQOLfBI+cWYTHa+fKEo2xXQ8GA0yDYSybwmql1vax47ZPTeTvKLYPG/hST92NzPGipZ6yb79McHcycUu1fcL2Eetf579ErR16Col/Exw5LaZDJbUjHnYur1bxri8+joqtieqViFV4/rW2jxQ51fN3PR50sizbLgppsR6x4QrfAYj8w9kE4fmsjfx920eJ/NuN0GWpp5ynIL8Of9c61Pk7HhbLFgCQ598jSPyb4KHTYi7soA4K6ZQHTs3jEz86g+MXV2oeUxu2jWRT2lLPIOFrRlsLyO1ly0VeTfgOeJ2/7kZmx9o7BLaPb4E5Hm/bm5e2T9bUWz3dGuPYLDLhL1s7kOffG0j8G+C4Hh4m8a+LjFB158dWGraNJSR845F/MNDDt9lKSrVPvONn8BoDGfmr1T5e0M4ZCAe5B3ZXF2yfyGKv9bZ9XC/s60O2T08g8W/A0QsrKFpS3Mj20VHVVKdIbGVxkrR9ZHli/OdV20fdXopF/rqEr1xJPKief7xfvzxX8ngtl7c1vxcQ59Q0WETwDYNp/f+1QCb8ZUdPivx7A4l/A448J/z+A1tGBiKq7AVyMZLu5mi7XrDydCSbgsfDsYySIPKP1ZjL7WU7Wucv2x9EXsMXQY+HU6/6FXnTc5TjFJ6/ushLin/4eKaDksxcyqjp3qkOi19LMjHPnyL/3kDi34AjpxewcyKHA1tGyPZJoKqxKYLH1ISv35M9bv2Ei7yiyUZ5MylZTuBLyxtJPMKPR8n9TLStQ3gjSOkSvkqVUyciLWclqOj6/KwFgfiXbJgGw2iWevn3AhL/OnDOceS5BRzev0nUHlOpp5aqpjpFYisJ3zH/TRxP+oaRfzThW3U8eB5HxfaQz6T850STwcHv0Qhmv6K2dZDnNB75S6GXxy1aOrf/ds6matsmy3PdrQHuzRJ4/mUbk/l0UIFEdBcS/zqcWyzjwnIFh6+Y8mvPyfPXESR8dZ6/kvAdySZE/nHPPx16/hUn7OWvPseOCbz6qUO2R+hXtJG/y6P9/INFXqLFQyctnQGxyjcp8l8Pz992ORZLFi3w6iEk/nWQJZ6H90/VtBkmQqTo66t9oglfQGf7CLGO2wxVx40McgHUiDdq7USbnPW57aPeyJREdio2yQsQN0E5zjFttB8hi3bZMfEPPP+1j/wB4NJylfz+HkJmWh1OzRYBANdsH6uZLkWE1LN9LMWuGMs1sH2UFb7ydeUIx8Dzj9kd8deIf92PJIm/btWt7fGwhUYHts/Lr94WVFRJMikDaZOtue0ibb9LK1Uc3Dqypr97mCDxr0PFcZFJGUiZBkX+dahn+1iOF4h+ou3jR+rxap+q7UV6+QNhFFrr+UeHmfczuhtZfBFXWon8ZYuHTuyZ33v1NTXbhPivvTkQin8FL7xias1//7BAtk8dqraHnLLwaJg9/9mVauLxV+raPuFs3pGsiCyLjTx/ZYVvyRLPzWf0ZaDqa5i+7dH3kb+rRv5hwlet9pGlno7nBS0eOin11JEx10f85XVQsT2yfXoIiX8dKrYbrn5MmbDd2vryOPc9eh5/98Dptdi9NYNzjle//1/xdz/UH5cUKF1vH9WukNF7Up2/OsAdEFUvoe0T9vaRrxv/PSO+bdHvLR7ikT/nXOROFE8/E5S8KrZPl4U6vc7iD9ACr15C4l+HiPin9RFnnM8eOYtPDJj42y7HfNHCxeWK9vF6df5qwjenRHQqSYu8qo6HSytVAOFCnyTxrzpekFDu98i/GvP8ZcARn+ELiPMgbZ92V/gmkUkZXf800dTvNcPcA0X+vYPEvw4V2wsGXYSCVN/6Wak42gi4n5GRejxil4TVPvpe/emgXtxAymC1kb/rImUwGEZ0kZfleLj/6YvYMprBNTvGAIQCaDm11T4yp9Dvnn81FvnLfEZkkVcqbOxmBZF/l22flLHmU7zk75VQ5N87KOFbh4oTtX2AxlFlseokimS/IhcdxSP24PE6jd0s14t0hcynzZqbo+V4kTe8PNcrFQffPjaL1928M/Dz5WvFrR1LEf9+j/zj1T7S048kfI3wPATzELoc+e+dKtTkZ9aCiPhTO+eeQeJfh4rtIhebbNSo4me16iSKZL/SMPKv19gt1mc+mza1to+uadm3j1/CatXBq2/YET7mWwLxRV6WYvusVeT/jk8+jDuv24433rK7q69ruR4YAzgXn4pk22btJC/X0z7eDd712uvg8bVfM6FeL7TIq3eQ7VOHiu0FXn+zts/qAEb+QTVPI/FPaO+gilIubdS8juVGI3/T7yj54zOLGMmYuP3g5uCxpN4+wvaJdgPtNfc/fRE/evZy11/XctxI/kI2qtMu8lJtny5bNOLvsA6lnkprafL8eweJfx2i1T76ZKUK5xzFqgPL70kzKDSO/Ov184/2nMmlzZrXqTq1U6jk9y+/dltszGBtbx/PE9Uwa2n7yJ5DRc1M4k6xHC/og1R1vFDcNe0dbC+0fTpZ4buRUK+FSRL/nkHiX4eq4ynVPo2jyrLtQmp+ZYDWBFQaef4Jdf66njNJnn82FrXK8/3T12+PbM8E1T7Koi7fEw9tn97feNVZA93Gcj2M5YToichfJnw1jd2c7qzw3Uio1wJF/r1jMK6WHiE8//qLi1TUtgWD5PuH4t+a7SNFOJuK2j7xc2PHbB/5M2mT4RXXbotsz2j+DvLrMPLv/Y1XfnqRi9C6ieV4QfvrquPBCRK+UWuMMX+RV4/q/NeLYPJbNtX18lUihBK+ddDZPvU8/xWlMmKQfP9Ks7aPRtSBaJVKLm1iRdPbJy7+47k0rtkxhvFcNPLT1fnLm8xaRv6h+PfG9pkoiCqXquMFZa3xUs60afjVPvrH+xV5LYxT1N9TSPzroNb5N1Pto5bFDVKtf5jwrT12znliewddRJpLm5j1F25JLLfW8//QL70g6AmkEh9iAoSRf5AkXYNqH7nyuBeRf9W3wWRLERn5qzN8AeHxO8oK37UeutIr5HFQjX9vIfFPgHOurfOvJ/6q7VPuQUS4XtRL+Or60MQfi4t//BzqIv+DW0e1+6JL+Erxk22f1yLhK2/uvUr4ZlMGsv4s2yCyj49ZTBlwlDr/QbFI5LVA4t9bBuNq6QGW64Fz1No+dSJ6tVvlIDWBq+f5q0LeqG0DIFo8xG+MOvFPIpjkpWl7LFsQr0nk38AK64Sqfz7kOMOkap6UYcByudLVc7BsH1rg1VtI/BOQQhY0G0vXRpxxVPEvW4OT8FWFjscW/UgrqJAxExO+qrDnM2ZNJZSu1DMJw2BIGUxr+2RThpgCtQaRv7yB9WIFrOV3QpVtxB1NewdAdPF03LAUdNBsH1rg1VsG42rpATLCr7F96lTxDLrnL1acxhuqieOcyKdR9TtQSpI8/5pST021Tz3S/pg/3e9J+/Nfe428IaqN17qF5d8Mg8hf094BEDcD2w0XgQ1KtQ9jDLsn84nWH9EdyPNPQApebbVPsrAMerWP+NqLDPqW52M8l8bMUgW2y8P+O46m2iclSj0558GEKKuFyF++XqTU0w1tn4wviL1GPSclywnq8ruBtMGyKfFpynb04p42mT/JS58T6Ge+8bs/OTA3s40Knd0EpDURVPsoc2WTGNzI39V+DYSfhMbzsi49fFy3+CiXqU2ct+L5A0LkI6WeijimzbWZuKbmLbpd7ik/CcnIX87oTWlKPW1lBXBqQFb4AiLoMgfoeDYiJP4JSJGTjd0MgyHTQFiii7wGR/xVoasRf1/sZT1+XNQBIBtJ+Jo1r9Oq7ZPxq2CCfVAi/2zKWJNqn7LdG/F3XGEjZUwz8PyDap5YqWfKZHC83g1zIQYbuloSiNs+gLB+6nn+K1UnKE8bqBW+ipjq+vIA4YIcVfx1doQ8n+r5sVu1fRIifzl2cC1sH/U8dDPpK6P4bFqJ/GMzjiXpwPPnMA1GkTLREl0Rf8bY7zHGOGNsi7LtnYyxE4yxY4yxVyvbX8gYe9x/7INMGr8bjCDyVzoMZtMGLLe+7bN5RJSnDbLnrxJ6/r7tY2tsn1hXTyA8P2I9hRe5yTYinvCNeP5rFPlXlGi/m39rtTw2E4/847aPIcRfjMrckG8jYgPTsfgzxvYCeBWAM8q26wHcA+AGAHcB+BBjTL67PwzgXgCH/H93dboPvaASq/YBxBuyXuS/WnUwWcggbbKBsn3UY4nX6MvHdJG/bsJUPh21fWS1jOzL0wyyrYFEXeEq8gFr194B6HLk79RaWHaS559iQUtnsnyIVunGFfM+AL8PQH3H3Q3g05zzKuf8WQAnANzKGNsJYJxz/kMuagI/DuCNXdiHriOtjmjkX7s6VWW16mIkm9K2Le5nKrYbrJ7V1egD9T3/jKmzfcTryLURo9nmI/94dB9UFclFXmsQ+Zd6lPBVF6xlUiYspZQzbvukjHCFL4k/0SodXTGMsTcAOMc5fzT20G4AZ5Xvp/1tu/2v49uTXv9extgRxtiR2dnZTna1ZaQ4qWWNWb/XShKrFRtjvvgPkudftt2gr3p8hXM1iPyTbZ/IiMaY7SOj5lYi/4wZW+SleOJSMHtNrxK+gecvF3nZbmL7Bml/2Q4n24domYbvOMbY/QB2aB56F4D/AuCndT+m2cbrbNfCOf8IgI8AwOHDh9d0OorO9pHVF0kUqy5Gsqa2Z30/U7E9TBYyOL9USUz4yr7rFafWjon38wfCElEZ+Rcyrdk+uhW+GdNAZo0i/4rtYqqQxkLJ7mpzt7jnr3btjJdypv2boO1R5E+0TsN3HOf8Tt12xthNAA4AeNTP2e4B8DBj7FaIiH6v8vQ9AM772/dotm84tAnflFm/n3/VwWg27fesHxzxL1sutm/JAqiT8NV8MrA0s2Xjto9sjDbaoudfVKJtK2KVrFG1j+Vi82jWF/8eJHyDyD95Rm/aNPxSTz4wrR2ItaPtK4Zz/jjnfBvnfD/nfD+EsL+Ac34BwH0A7mGMZRljByASuw9yzmcArDDGbvOrfN4K4EudH0b30ZV6ZupE/p7HUbQcjGbNgfP8q46LSb+/fDzhW6/O367j+dfaPq15/rb2EwarSQb3irIf+TMGlHpQ6ilvZFXf0zcYako5U/6nnPicZIJohp60d+CcP8kY+wyApwA4AN7OOZeq8esAPgYgD+Cr/r8NR8V2awZYZ1MGFsuW9vkl2wXnwGgupe1f08+ULSF0gCbhK1f46hK+Gs8/H6vzDxO+rXj+UYG3HCGOKXMNG7vZHibyaRTSZuRTSKdUg4aCZvBJ03Y9bbvmjGkEk7zilUAE0Yiuib8f/avfvxvAuzXPOwLgxm793l5Rsb1ghKMkm04u9ZSre2W1z1LZ7vk+rhUVx8N4TkS5lZrIX6zOlfZYpL2DrreP/7xKBwnfdCzhq1a7rFVjt4rlYsd4FvlMqssJX/Fa0vYBREJZZ+ukTCr1JNqHrpgE1EEuEtFoK0H8lQg2nzbq9v3vJ2y/3UA+bSKXMiMJXUCIfdZvQgZEu57argcWsysCz9+Jlnq2WuevRvfyBgSgYQuOblG2XeTTJkayZu8Svr6gF6uONrJPGeEKX/L8iVahKyYBdX6vpF6ppxSxsdxg1fnL48hnTOTStYNYxMhBMyjhjNo+HGnTgLqIW0azFSua8B3JtLDCN2UEyWTAHwBvhuM216q9Qz5jotCFyP9zD03j/GIZQLTOX57TouXU9PWRzwlW+KbI9iFag8Q/gartBW8+Sb3WAYF9kUkNVKlnsN4hrS9hrdpeMEQFqO3qGY9IGWOiGso/jyXLQS5ttDSCUDR2i1b7qJH/WrV3yKVNFDKdRf7FqoPf++yj+OwRsfwlPphGPMfV1vGnlBm+ZPsQrUJXTAIV2w06UErq1fmv+J6/TPgOygzfij+RLJcyRCI7dvwVx0U2bWi7nia1alYT4qI8trXUU7yFg610Bc2kDHgcXR+wEkfaPoWM2dEcX5kbkv+rSXIZfKxWHa24y1LPquNpPxkQRD3oiklAeP6xhG8dz7+oeP46kexXpDcvbJ/am1pVGe4S73qa1HAsr7xOseq05PcDtQlfNeEZDHjv4fm3XdFjX4p/Jzf6GvGPeP7ivCZ5/vLcVmw3GKBDEM1C4p9Axa7tNJlNGXA9HvRaUVmNiL+wHnodfa4FUthyKeH5x3MeMuEL+NVQqh2TYEeoN8fVqtvS6l4gjHg9//yqk8DkJ4BeVvyoeZCRTArFDmyfZV/0lysx8Y9V+6Q1kb08tyXLJduHaBm6YhLQJnw1SU2JWrUStDCo0weoX6jYDSJ/xwvFP/bJKGnlaTZlREo9W2nqBoQCL2fbWi4PZgZkzOgIyV4gk9X5jIl8lyP/qur5pxTbRxPZp0j8iQ6gKyYBIf61tg+gFxbhyzJkfW8cqF0N24+UlTYX+bSp7eoZzDmOtbWwHLeh51+0Wrd9MjFrx3LcYFpYcGNYi8g/bWIk22Hk7+eKlhXbRy5YkzfVYjWh2se/0ZUsfU6AIOoxNFfMhaUKfuGvHsB8Ub9CN45Y5BWNSKWwaCP/ihAxxli4inUAfH+1zYXe83frRv46UcrHEr6te/5S4Hn4e/zIeC08f1X8CxnRwbVdi09G/MtKwldNXgOA4+m7dsrI3+PQPk4Q9Rga8X/4zAJ+cPIynji31NTzq46LrMbzl4/FKSpVK0Hb4gGI/KvBIHtT26racrzgPMWroZISvqLxnXhesepgtA3PX76+3Ie459/NyN9xPfzyX/8IPzg5B0DJg2TMYM5Bu+s6Qs9ffALQHQugn8+rdvmkyJ9olaG5YmTEv1BqIfJPsH10kf+KIv7xaVX9TCB06aSEr+r5R1c2WwkNx9RFcEV/AE4rpGO+vq2p9unmKt/LRQvffWYOD5y8DCA8JyLyF/vebnM3mehdrTpwXM9frSxvpmHwoe3t0+DmQBD1GJorZtEX/YWmbR/9Cl8A2v4+auQfb1vcz1QUiyOvTfiq1T5x20df5y9tH87DTqitEK/oiSzy6kG1z+VVcc1c9q+duO0DtD/QRe0BtVJxxCcp5WYqSWuGs6t5gAzZPkSLDI34zxfFm2y+1LjhmuPXcdcs8tI0L5OsVh2M+kPM85lo58p+phx4/uEiLzGBU1BT5x/r6qmt9vHto5IlOqG2m/ANbB+1vYN8rIuRv/y0KP9XSz1l5N9u0ne5HP7ccsWOeP6q+Ner8xdfD81bmegSQ3PFLLQQ+evm9wK1VSYqauJS3jQGob9PMNTGr/N3/eEhweP+Cl+gtveRGC+os32EPSQFs+2Er6PU+fcw8peWofxftX3kHIJ28zvLSuS/VLZFhVSTnr+6rZX2GAQBDJH4B2/gJjx/3QhHAEFiM6naZ0x6/plo2+J+RqweFe0b4h055Y0gUu0TX+GbYPuUbbetKV4AgteUAh9p7xD7VNAN4uKvXh/S9mm3p/9yxcamETEoZ7nsiBxKOjyfkkbiT9U+RKsMjfi3FPlrRjgC9at91Mg/O2CRv0xgB+LvC13YhCys84/bPvpqHxOOxwO/u2Pbx+lte4dQ/MX+6myfdhO+S2Ube6fywddJ1T7x+b1A1ArS5VYIoh5Dc8VI8W+mzl83whFQxT8qLK7HUbLcsNonI3vb97/4l5XFbvEpXPImGKn2iXX1zGpESb7e5dUqgNbaOQMI+tgEi7w0tfFqy+dOUSvFPI+jrDS76zThu1y2sWdTQXxdsSMWlmmwYBaC7hNUNPIfmrcy0SWG5opZ8KO2xSYSvkEb45qEb+3AEiBM9sWrfQYj8vdqI39f4KtO9CaZi1X7JJV6yteTVTTtL/ISyWd1MVm9vEy7SKvQ9ThWKg7KtvDlU6YRRv5tJHxt10PRcrF3Soj/Ujma8AXCG6uu2ocSvkQnDMUVU3VcrFYdMCbeyGq1StLzgeSEbzXmJwcdPXMy4Ss9//6r9rEcD+/45MM4dmEFgIz8pbhHF6+F82bDyN9yvKDhWtIKX3kTnSv6kX+L4i/3p2S5ge+fjUX+3fT8VatwvmRFWn90EvnLNuDbx7NImwzLMdsHCI9Hl9Alz5/ohKEQfxnt750qwHK8hm/URNtHlnrGInp1fi8g3qhpk/Vl5P/sXBFffmwG3zx6CYD4FCTFOr54LbB9YglKSynBTFrkBQBzK0JUW034bhvLAgAurVSDyiMpfvEFYN1gvmgFYj9frKJsuYG1J89JOwlfmfOYyKcxnkuHnr8u8teKP0X+RPsMxRUjPduDW0ci3yeRWO2T4PkHIxwVEVObl/UT55fEOMEL/v9V20NeinvMzqrGE77KIjhhx3jaxUfyk9HlIPJvzfOfyKeRSxu4sFSO9L8HehP5zxctHNw66n9to2yHbagNg4lpXm0kfJcV8Z/Ip7FccSLziIHwePSTvMjzJ9pnKK6YhUD8xRu4UYuHil2/zj8u/vKTxXi+/8X/wlIFAHDe/1+1fRomfJVFcI7Hwbm+CkVGzYHn32JvH8YYdk7kMbNUCUQ+HYuQu9XegXOOhZKFq7ZJ8a9GzgkgrJ9SG39rGfmP59MYy4vIX22XAYQ3Vl1XTzUJTLYP0SrDIf6+OB8M3sBNRv6xhC9jTDvEfdofvr17shBsy2uaoPUDM4sy8hfiHy31jK5fqPX8w3UQgSjXs31WqyhkTBiaZGYjdozncGGpUhv5dznhu1J1YLs8EvmLcxIeVyGTai/yr8Qif3+Rl1poII9Hu8JXOW+6ldQEUY+huGJktcaVW4Tt06jip+LobR9AP8R9eqGEtMkCL1r8rNGXXT1nfNGf8W2fSOSfiXr+lcDzj9k+jhusvtWKf0qKv9VysleycyKHmaVKZOYtIGyY+JjHTpCfGndP5pFLG1goWRHPH/Aj/w48//FcGuO5lBD/eLVPOnpTU6EVvkQnDMUVI9/AB+p4/p7Hg57sSbYPoJ/je26hjN2T+UgEqxt80g9I8Z9btVB13Mg4y3jbCl21DyDOnxWzY1TkCuj5YrXlZK9k+0QOl1YqwT5kYjXv3Yr8ZTO3TSMZbB7J4vKqFQxvl7Qr/rKvT+j5a6p96kT+qUjCl2wfojWGQvznixbGsilsGcnCYHrP/333H8fr/9/vAUhO+AK1Q8oBYHqhjD1ThejzNB0w+4GZpTKYryMXl6qRssZcjecfE/90re2T1ZV6+jcRj7ee7JXsnMjBdjkuLItPKPEkabcj/6mRDKZG0iLyj3n+7U7zWirbSJsMubSB8XwaCyUbHq89FqCZUs+heCsTXWQorpiFkoWpkQwMg2GqkNFG/g8+O4+nZpZRrDpBKadudWp8SDkgxT8f2ZZXhpT3C5xzzCxVcMjPjcwslSOtrcPIPl7qWWv7yMhbN3s2IpwtJnslO8ZzAIDTl0vi98Qj/y6Jv4z8N49ksGkki8tFCxUrGvnrWl03w3LFxkQ+DcYYxnPp4JNnRpPw1VVNpTXrAQiiWYbiilko2Zjym2dNjWS0kf/J2SIA4NRsERW/4oKx2jdc3Pap2C7mVqs14p9LG0EPnH5hueKgZLl4wb4pAMDZhTIcjwdCZxgsMnw9WOGbin4yaJTwVf3y9j1/cb6l+EeiZdOA5XSnvYMa+W8qpLFQ9G2fTHci//FcGoCwfiSq7SNvqLpqH9NgkE4jRf5EqwzFFbNQtLCpIN5cU4V00OpBslSyMef3mTk5u6od5CLJxHrWTy/4lT7ayL+/xF8meW/ZNwkAeHZuFUA095HPmLXVPvHIX/X8tQnfcFu74r9jQkT+Z+drI/9MqnuR/3zJQiZlYCRjYmpEfGosxSP/TJuRf9nGuC/6apmwbpGXzvMX2+XNgTx/ojWGQvznixamCn7kX6iN/E/6Igeo4q8/NaKFQfhGP+eXRsY9f92w842OTPZetW0U47kUnpsTwqoKXS5lKou8ahu7ye1n58V52apUQElSphGIVatTvCSbRzJImwynffHPxiL/bg1zmV+1sKmQAWMMm0cyWK2KhVhR68oM2lO3gir+kchf4/knlXLKck+yfYhWGYorRnr+gKjaiHv+Jy4J8c+nTV/8vcTIPz6tanpBiE+t7dN/i7xmFoX475jIY+dEHqfmhBWmDrJXh69XHQ8GC6NOtfHdE+eWYBoM1+8c1/4ueUNp1/M3DIbt4zlt5J9Osa5F/gslK+i3L68hIGpd5TMplG036GnULMsVJxB9af8AiC3ySk74ArWL2wiiWQb+iqnYLkqWG3kDL8Sau52cXUXGNHDblZtw8lJRRP6pJPGPDiyZXij7Nf65yPNyCYu8Hjh1OTK3dSNxYakMg4neOTsnc4HtE4n802akvUM2ZQa5ETXyf+L8Eg5tG02+iUrxb9P2AUTFj7wRxz3/blX7XC6G4r9ZFf9Y5A+03sVVeP7i+NXIP6ut9kmwfYzk9g8EUY+BF3+5oEvaPpsKGdguD/rxAMDJS0Xs31LA1TvG8OxcESWrju0Tq/aZXihj12Q+6LsuyadNWK4XVHAAwMXlCt7yVw/gQ9860bXj6ybnlyrYOpZF2jSwcyKnbXCnfqKp2uEIRyBa5//EuSXcsGsi8XfJ89tunT8gPqFI4k3OutXeYUERf3kNAVHxL2Rbn+PLOcdy2Q4j/wa2T1qT8AXCKiCK/IlWGfgrZj5YpOMnfP03spr0PTW7ioNbR3Fw6ygs18Mzl1YiVoeKzvaJWz5AbSsEAPj+iTlwDnz3mbkOj6o3XFiqBFU0OxVhjZc1VhXbR9eH5vR8EXOrFm7arbd81NfsJPLfMR7mE+KC2a1FXvOK+G9SIv+cYvts8bdLC6oZSpbofxQkfHNKwtcMX1ue06TIXtpBJP5Eqwz8FSOTu5NBwjcd2W45Hk7PlwLxB4CLy9W6nr8qLHJ1b5x4KwQA+N4JIfpPzSwH1UUbifNLZez0q2hkNQ0QrfbJpY3A3qjY0T40aZOBMeCh04sAgJv21Iv8pfi3l/AV+xie92xEMLtj+9iuh+WKE35qTLB9bj+4GabB8K2js02/ttrXBxAiLu2jZhd5AeKcGww1nzwJohEDL/7zyvJ8IIz8Zb+f05eLcD2Oq7aN4ipf/IFoOaKKWudfsV1cWqnWVPqIn4/6wJxzfO+ZORzw+wv94OTljo+tm3DOI5H/LkVYc7GyxkrE8w/Pk2x8d+zCMgwGXJeQ7BWv2bnts1O5QamLybrV3kEGCJtGxTUzWcgEq58LSuQ/WcjghVdM4Rv+DIR6yHMnWzuoiV55I9D389eLe9o0KOon2mLgrxr5BlY9fyBcvHNyViQ1D24dxUQhjS2jwkqoX+0j3sDngzJPje0TRP5ChE5cWsWllSp+7aUHMJ5L4fvraP187ckL+PaxqFAtl8UCL33kn1Tq6UU8f0DcHD0uzmehTiVPrhu2j7KP8elX3Yj8pTUorxnTYJj0BTp+fbzy2m14emY5uCZ0nLi0guf9t6/hvkfPRwa5SKQFpJvklSTwadOgjp5EWwz8VSPfwJOFqOc/H4i/KGe80m/6Jge+1KvzrzpiWIlc4KWP/KOev7R8XnZoK+44uAXfOzHXcJxkLzi/WMY7PvVjvOOTPw4GqAPAjN8jZ+ekENSdCbZPVqliqsbaDwNhpHrT7mTLR7ym+LluRP4sZnt0K/KXw2ZUu0deP/m4+F+3DQCCCWg6/uybJ1B1PLz/68eDoEQr/po8SrLnz7TN8wiiEQN/1SyULIznUkHkNJ5LwTRY8OY7eWkVOydyQQQqe/7XK1HkXMynDcW/sef//RNz2L+5gL2bCnjJoS04t1jGc5ebTxB2iw/c/ww45yhZDv78WyeD7bLGXwrqSDYVJCHjCV91hW/8JinP241Nir9qn7TK1lHRqC9jRltxiBW+nd9Yg8hfEX/5KSAf2++DW0dxxeZCovg/N1fEfY+ex/U7x3FqrohPP3gGQHRlr7SAVCvt8BVTuPO6bdoAAxA3OlrdS7RDR+LPGPu/GGPnGGOP+P9eozz2TsbYCcbYMcbYq5XtL2SMPe4/9kGma6DTRdRqDf/3Y6qQxrz/xj4xuxpMaQIQ+P6J7R2CSVEuphdKSPmLjeLIny/bLmzXwwOn5vHiq7YAAF7i//+9Z5pPEHaDE5dW8NmHzuKXb9uPn3vhHvz9A6eDRWpyda9a5bPLT2RHSz2NmOevj/wbin+qc88/ZRrYNparsT1Eb5/OF9jNayL/TQmRP2MMP3XtNnz/xJx2ZfeHv30SadPA3/y7n8BV20bxrWPib69G/vJrVfz3bxnBR9/2E4nXY9pk5PkTbdGNq+Z9nPPn+/++AgCMsesB3APgBgB3AfgQY0xevR8GcC+AQ/6/u7qwD4kslKyg0kcyVchg0V/odfLSalDlAyiRf1LC1490LcfDucUydk7mtJUW6sjDR88uYrXq4KWHhOhfsbmA3ZP5wApaK97zL8dQyKTwjp+6Cr9159UAA9779eMARF8fucBLIj31XCzydzwxn1fYPjHPP22AMeCGXcnJXiCMnDvx/OU+xlsbCM+/88h/PmYZAsniDwCvvHY7qo6HH5yM/l2nF0r4/MPTeMut+7B9PId3vOKq4LGxnGr7pIL9b5aUYVBrB6ItOnvnJXM3gE9zzqsAnmWMnQBwK2PsOQDjnPMfAgBj7OMA3gjgqz3aD8wXrZrIfGokg+8cn8Wd7/0OipYb+PxA6PnXq/MHgJ/7ix/i4nIFz9szqX2etEP+z394Ah7nMBhw+5VC/BljeOmhLfjCw+fwqvd+p6PjaxYOkXT+nVddHQjYr9yxH3/13VN4fHoJl1aq2DaWi5QU7pzII2MakZubvBHc9f5/xZn5Uk1FTzZl4sotIw1FPZc2wVhnto/Yx1wwclKSNhnKttvxuZ1drUYsQyAU/3iiGwBuPbAJIxkTv/+5xyKfFpbKNhgD7n3ZlQCA1928E++7/zgur1qRc6ur9mmEqPYh24donW6I/zsYY28FcATA73LOFwDsBvCA8pxpf5vtfx3froUxdi/EpwTs27evrZ277crNkeQlAPzqi/fjvkfPAxCJyVddvyN4bPdkHr/7qqvxMzfugI6XHNqKNz5/FyzXw3U7x/CmW/Zon7d/8wh+6bZ9QWL5xt0TmFAiyLfdsR+rVQfeGiZ9X3LVFvzaSw8E37/9FVehbLm4XKzi0PZR3HFwS+T5v/iifUFvf8krr9uGx88twfE8XLNjDG8+vDfy+H946ZUQt5r6vPH5u7F9PKdtm90Kv3LHfrz8mmJk22tu2onTl0sdn9tD20eD9taSN92yG2O5dOKIz3e99np870Stnffiq7YENlrKNPCnP3sznppZjjzn9c/bhbRp1K2SivO2O65oOJOaIHSwRhUnjLH7AeiU8F0QAj8H8W7/YwA7Oee/yhj7cwA/5Jz/vf8afw3gKwDOAPgTzvmd/vaXAvh9zvnrG+3o4cOH+ZEjR5o+MIIgCAJgjD3EOT8c394wxJBC3cQv+CsAX/a/nQaghoR7AJz3t+/RbCcIgiDWkE6rfXYq374JwBP+1/cBuIcxlmWMHYBI7D7IOZ8BsMIYu82v8nkrgC91sg8EQRBE63Tq+f8pY+z5ELbPcwD+IwBwzp9kjH0GwFMAHABv55zL+rdfB/AxAHmIRG/Pkr0EQRCEnoae/0aBPH+CIIjWSfL8qUCYIAhiCCHxJwiCGEJI/AmCIIYQEn+CIIghpG8SvoyxWQCn2/zxLRCL0QYBOpaNySAdCzBYxzPsx3IF53xrfGPfiH8nMMaO6LLd/Qgdy8ZkkI4FGKzjoWPRQ7YPQRDEEELiTxAEMYQMi/h/ZL13oIvQsWxMBulYgME6HjoWDUPh+RMEQRBRhiXyJwiCIBRI/AmCIIaQgRZ/xthd/gD5E4yxP1zv/WkFxthexti3GGNPM8aeZIz9pr99E2Ps64yxZ/z/pxq91kaBMWYyxn7MGPuy/30/H8skY+xzjLGj/t/o9n49HsbYb/vX2BOMsU8xxnL9ciyMsf/FGLvEGHtC2Za474yxd/p6cIwx9ur12Ws9CcfyHv8ae4wx9kXG2KTyWEfHMrDi7w+M/3MAPwPgegBv8QfL9wsOxFjM6wDcBuDt/v7/IYBvcM4PAfiG/32/8JsAnla+7+dj+QCAf+acXwvgeRDH1XfHwxjbDeA/AzjMOb8RgAngHvTPsXwMwF2xbdp9998/9wC4wf+ZD/k6sVH4GGqP5esAbuSc3wzgOIB3At05loEVfwC3AjjBOT/FObcAfBpisHxfwDmf4Zw/7H+9AiEuuyGO4W/9p/0tgDeuyw62CGNsD4DXAviosrlfj2UcwMsA/DUAcM4tzvki+vR4IOZ65BljKQAFiOl6fXEsnPN/BTAf25y073cD+DTnvMo5fxbACQid2BDojoVz/jXOueN/+wDCSYgdH8sgi/9uAGeV7+sOi9/IMMb2A7gFwI8AbPcnosH/f9s67lorvB/A7wPwlG39eixXApgF8De+jfVRxtgI+vB4OOfnAPxPiPnaMwCWOOdfQx8ei0LSvve7JvwqwuFXHR/LIIs/02zru7pWxtgogM8D+C3O+fJ67087MMZeB+AS5/yh9d6XLpEC8AIAH+ac3wKgiI1ri9TF98PvBnAAwC4AI4yxX1rfveoZfasJjLF3QVjBn5CbNE9r6VgGWfyThsj3DYyxNITwf4Jz/gV/80U5O9n//9J67V8LvBjAGxhjz0HYbz/FGPt79OexAOLamuac/8j//nMQN4N+PJ47ATzLOZ/lnNsAvgDgDvTnsUiS9r0vNYEx9jYArwPwizxcmNXxsQyy+P9vAIcYYwcYYxmI5Mh967xPTeMPuP9rAE9zzt+rPHQfgLf5X78NwJfWet9ahXP+Ts75Hs75foi/wzc557+EPjwWAOCcXwBwljF2jb/plRDzqvvxeM4AuI0xVvCvuVdC5Jf68VgkSft+H4B7GGNZxtgBAIcAPLgO+9c0jLG7APwBgDdwzkvKQ50fC+d8YP8BeA1EhvwkgHet9/60uO8vgfgY9xiAR/x/rwGwGaKC4Rn//03rva8tHtfLAXzZ/7pvjwXA8wEc8f8+/wBgql+PB8B/A3AUwBMA/g5Atl+OBcCnIHIVNkQ0/O/r7TuAd/l6cAzAz6z3/jdxLCcgvH2pAX/RrWOh9g4EQRBDyCDbPgRBEEQCJP4EQRBDCIk/QRDEEELiTxAEMYSQ+BMEQQwhJP4EQRBDCIk/QRDEEPL/A05UqNm+29/qAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def play_episode(env, agent, max_episode_steps=None, mode=None, render=False):\n",
    "    observation, reward, done = env.reset(), 0., False\n",
    "    agent.reset(mode=mode)\n",
    "    episode_reward, elapsed_steps = 0., 0\n",
    "    while True:\n",
    "        action = agent.step(observation, reward, done)\n",
    "        if render:\n",
    "            env.render()\n",
    "        if done:\n",
    "            break\n",
    "        observation, reward, done, _ = env.step(action)\n",
    "        episode_reward += reward\n",
    "        elapsed_steps += 1\n",
    "        if max_episode_steps and elapsed_steps >= max_episode_steps:\n",
    "            break\n",
    "    agent.close()\n",
    "    return episode_reward, elapsed_steps\n",
    "\n",
    "\n",
    "logging.info('==== train ====')\n",
    "episode_rewards = []\n",
    "for episode in itertools.count():\n",
    "    play_episode(env.unwrapped, agent,\n",
    "            max_episode_steps=env._max_episode_steps, mode='train')\n",
    "    episode_reward, elapsed_steps = play_episode(env, agent)\n",
    "    episode_rewards.append(episode_reward)\n",
    "    logging.debug('train episode %d: reward = %.2f, steps = %d',\n",
    "            episode, episode_reward, elapsed_steps)\n",
    "    if np.mean(episode_rewards[-10:]) > -120:\n",
    "        break\n",
    "plt.plot(episode_rewards)\n",
    "\n",
    "\n",
    "logging.info('==== test ====')\n",
    "episode_rewards = []\n",
    "for episode in range(100):\n",
    "    episode_reward, elapsed_steps = play_episode(env, agent)\n",
    "    episode_rewards.append(episode_reward)\n",
    "    logging.debug('test episode %d: reward = %.2f, steps = %d',\n",
    "            episode, episode_reward, elapsed_steps)\n",
    "logging.info('average episode reward = %.2f ± %.2f',\n",
    "        np.mean(episode_rewards), np.std(episode_rewards))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
